{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Build Sample Data\n",
    "\n",
    "This tutorial shows the details of sample data building. This is exactly what we did in `xenonpy.datatools.Preset#build`.\n",
    "\n",
    "\n",
    "### dataset\n",
    "\n",
    "We selected 1,000 inorganic compounds randomly from the [Materials Project](https://materialsproject.org) database for test and benchmark.\n",
    "You can check all **MP ids** at `mp_ids.txt`.\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### API key\n",
    "\n",
    "Before starting, users have to create their own `API key`. See [The Materials API](https://materialsproject.org/open) to learn how to do it."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# your api key\n",
    "\n",
    "api_key = ''"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### import packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from itertools import zip_longest\n",
    "from pathlib import Path\n",
    "\n",
    "from pymatgen import MPRester\n",
    "from tqdm import tqdm\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### fetch function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def data_fetcher(api_key, mp_ids):\n",
    "\n",
    "#     print('Will fetch %s inorganic compounds from Materials Project' % len(mp_ids))\n",
    "    \n",
    "    # split requests into fixed number groups\n",
    "    # eg: grouper('ABCDEFG', 3, 'x') --> ABC DEF Gxx\n",
    "    def grouper(iterable, n, fillvalue=None):\n",
    "        \"\"\"Collect data into fixed-length chunks or blocks\"\"\"\n",
    "        args = [iter(iterable)] * n\n",
    "        return zip_longest(fillvalue=fillvalue, *args)\n",
    "\n",
    "    # the following props will be fetched\n",
    "    mp_props = [\n",
    "        'band_gap',\n",
    "        'density',\n",
    "        'volume',\n",
    "        'material_id',\n",
    "        'pretty_formula',\n",
    "        'elements',\n",
    "        'efermi',\n",
    "        'e_above_hull',\n",
    "        'formation_energy_per_atom',\n",
    "        'final_energy_per_atom',\n",
    "        'unit_cell_formula',\n",
    "        'structure'\n",
    "    ]\n",
    "\n",
    "\n",
    "\n",
    "    entries = []\n",
    "    mpid_groups = [g for g in grouper(mp_ids, 10)]\n",
    "\n",
    "    with MPRester(api_key) as mpr:\n",
    "        for group in tqdm(mpid_groups):\n",
    "            mpid_list = [id for id in filter(None, group)]\n",
    "            chunk = mpr.query({\"material_id\": {\"$in\": mpid_list}}, mp_props)\n",
    "            entries.extend(chunk)\n",
    "\n",
    "\n",
    "    df = pd.DataFrame(entries, index=[e['material_id'] for e in entries])\n",
    "    df = df.drop('material_id', axis=1)\n",
    "    df = df.rename(columns={'unit_cell_formula': 'composition'})\n",
    "    df = df.reindex(columns=sorted(df.columns))\n",
    "\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [01:06<00:00,  1.63it/s]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>band_gap</th>\n",
       "      <th>composition</th>\n",
       "      <th>density</th>\n",
       "      <th>e_above_hull</th>\n",
       "      <th>efermi</th>\n",
       "      <th>elements</th>\n",
       "      <th>final_energy_per_atom</th>\n",
       "      <th>formation_energy_per_atom</th>\n",
       "      <th>pretty_formula</th>\n",
       "      <th>structure</th>\n",
       "      <th>volume</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>mp-20866</th>\n",
       "      <td>0.1849</td>\n",
       "      <td>{'Ge': 4.0, 'Rh': 4.0}</td>\n",
       "      <td>9.755532</td>\n",
       "      <td>0.039943</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[Ge, Rh]</td>\n",
       "      <td>-6.496651</td>\n",
       "      <td>-0.506775</td>\n",
       "      <td>GeRh</td>\n",
       "      <td>[[0.80283811 1.66009496 3.26577118] Ge, [1.660...</td>\n",
       "      <td>119.521991</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mp-30759</th>\n",
       "      <td>0.0000</td>\n",
       "      <td>{'Li': 1.0, 'Mg': 2.0, 'Tl': 1.0}</td>\n",
       "      <td>5.022910</td>\n",
       "      <td>0.027278</td>\n",
       "      <td>4.641088</td>\n",
       "      <td>[Li, Mg, Tl]</td>\n",
       "      <td>-1.970913</td>\n",
       "      <td>-0.099951</td>\n",
       "      <td>LiMg2Tl</td>\n",
       "      <td>[[2.85976352 2.02215817 4.95325571] Li, [1.429...</td>\n",
       "      <td>85.932461</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mp-3416</th>\n",
       "      <td>6.7145</td>\n",
       "      <td>{'F': 12.0, 'Na': 6.0, 'Al': 2.0}</td>\n",
       "      <td>2.844098</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-1.602530</td>\n",
       "      <td>[Al, F, Na]</td>\n",
       "      <td>-5.035544</td>\n",
       "      <td>-3.414627</td>\n",
       "      <td>Na3AlF6</td>\n",
       "      <td>[[3.6307153  1.31968004 3.40159567] F, [4.5775...</td>\n",
       "      <td>245.150290</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mp-505412</th>\n",
       "      <td>2.0103</td>\n",
       "      <td>{'K': 8.0, 'In': 8.0, 'S': 16.0}</td>\n",
       "      <td>3.080923</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.459451</td>\n",
       "      <td>[In, K, S]</td>\n",
       "      <td>-3.971909</td>\n",
       "      <td>-1.274151</td>\n",
       "      <td>KInS2</td>\n",
       "      <td>[[ 6.09352925  1.20936514 14.43416479] K, [ 6....</td>\n",
       "      <td>940.171218</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mp-684652</th>\n",
       "      <td>6.8002</td>\n",
       "      <td>{'Be': 3.0, 'F': 6.0}</td>\n",
       "      <td>1.246978</td>\n",
       "      <td>0.223526</td>\n",
       "      <td>-5.588439</td>\n",
       "      <td>[Be, F]</td>\n",
       "      <td>-5.541302</td>\n",
       "      <td>-3.346732</td>\n",
       "      <td>BeF2</td>\n",
       "      <td>[[3.76534884 1.64321    7.5707047 ] Be, [1.460...</td>\n",
       "      <td>187.798605</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           band_gap                        composition   density  \\\n",
       "mp-20866     0.1849             {'Ge': 4.0, 'Rh': 4.0}  9.755532   \n",
       "mp-30759     0.0000  {'Li': 1.0, 'Mg': 2.0, 'Tl': 1.0}  5.022910   \n",
       "mp-3416      6.7145  {'F': 12.0, 'Na': 6.0, 'Al': 2.0}  2.844098   \n",
       "mp-505412    2.0103   {'K': 8.0, 'In': 8.0, 'S': 16.0}  3.080923   \n",
       "mp-684652    6.8002              {'Be': 3.0, 'F': 6.0}  1.246978   \n",
       "\n",
       "           e_above_hull    efermi      elements  final_energy_per_atom  \\\n",
       "mp-20866       0.039943       NaN      [Ge, Rh]              -6.496651   \n",
       "mp-30759       0.027278  4.641088  [Li, Mg, Tl]              -1.970913   \n",
       "mp-3416        0.000000 -1.602530   [Al, F, Na]              -5.035544   \n",
       "mp-505412      0.000000  1.459451    [In, K, S]              -3.971909   \n",
       "mp-684652      0.223526 -5.588439       [Be, F]              -5.541302   \n",
       "\n",
       "           formation_energy_per_atom pretty_formula  \\\n",
       "mp-20866                   -0.506775           GeRh   \n",
       "mp-30759                   -0.099951        LiMg2Tl   \n",
       "mp-3416                    -3.414627        Na3AlF6   \n",
       "mp-505412                  -1.274151          KInS2   \n",
       "mp-684652                  -3.346732           BeF2   \n",
       "\n",
       "                                                   structure      volume  \n",
       "mp-20866   [[0.80283811 1.66009496 3.26577118] Ge, [1.660...  119.521991  \n",
       "mp-30759   [[2.85976352 2.02215817 4.95325571] Li, [1.429...   85.932461  \n",
       "mp-3416    [[3.6307153  1.31968004 3.40159567] F, [4.5775...  245.150290  \n",
       "mp-505412  [[ 6.09352925  1.20936514 14.43416479] K, [ 6....  940.171218  \n",
       "mp-684652  [[3.76534884 1.64321    7.5707047 ] Be, [1.460...  187.798605  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# read ids\n",
    "mp_ids = [s.decode('utf-8') for s in np.loadtxt('mp_ids.txt', 'S20')]\n",
    "\n",
    "# fetch data as pandas.DataFrame\n",
    "df = data_fetcher(api_key, mp_ids)\n",
    "df.head(5)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
